# -*- coding: UTF-8 -*-

import logging
# logging.basicConfig(format='%(asctime)s : %(levelname)s : %(message)s', level=logging.INFO)

import utils
import json
import preprocess
import lda_model
from gensim import similarities

class Web_APIs:
    def __init__(self):
        self.api_dict = {}
        self.api_category = set()
        self.api_user = set()
        self.api_arch = set()
        self.api_req_format = set()
        self.api_resp_format = set()
        self.name2urlname = {}
        self.total_count = 0
        print('start read api')
        for api in utils.load_json('apis.json'):
            self.total_count += 1
            if self.total_count % 5000 == 0:
                print('read api count:', self.total_count)
            url = api['url'].strip()
            origin_name = api['name'].strip()
            if utils.is_valid_data(url):
                name = utils.get_clean_data_from_url(url)
                self.name2urlname[origin_name] = name
                category = api['category']
                self.api_category |= set(category)
                developers = WA_Utils.get_developers(api['developers'])
                followers = WA_Utils.get_users(api['followers'])
                user = set()
                user |= set(developers)
                user |= set(followers)
                self.api_user |= user
                self.api_dict[name] = {
                    'category': category,
                    'user': list(user),
                }
        print('end read api')
        with open('api_format_data.json', 'r') as f:
            api_struct_count = 0
            lines = f.readlines()
            print('start read api struct')
            for line in lines:
                api_struct_count += 1
                if api_struct_count % 5000 == 0:
                    print('read api struct count:', api_struct_count)
                item = json.loads(line)
                if 'Name' in item and item['Name'] in self.name2urlname:
                    name = self.name2urlname[item['Name']]
                    arch = [] if 'Architectural Style' not in item else [item['Architectural Style'].strip().lower()]
                    self.api_arch |= set(arch)
                    request_format = utils.get_formats_from_string(item['Supported Request Formats'])
                    self.api_req_format |= set(request_format)
                    response_format = utils.get_formats_from_string(item['Supported Response Formats'])
                    self.api_resp_format |= set(response_format)
                    dic = self.api_dict[name]
                    dic['architecture'] = arch
                    dic['request_format'] = request_format
                    dic['response_format'] = response_format
            print('end read api struct')


class WA_Utils:
    @staticmethod
    def get_developers(developers)->list:
        if utils.is_valid_data(developers):
            res = []
            for dev in developers:
                if 'developer_link' in dev:
                    name = utils.get_clean_data_from_url(dev['developer_link'])
                    if len(name) > 0:
                        res.append(name)
            return res
        return []
    @staticmethod
    def get_users(followers)->list:
        if utils.is_valid_data(followers):
            res = []
            for fol in followers:
                if 'profile_link' in fol:
                    name = utils.get_clean_data_from_url(fol['profile_link'])
                    if len(name) > 0:
                        res.append(name)
            return res
        return []

if __name__ == '__main__':
    wa = Web_APIs()
    print(wa.api_user,'\n\n\n\n')
    print(wa.api_category,'\n\n\n\n')
    print(wa.api_arch,'\n\n\n\n')
    print(wa.api_req_format,'\n\n\n\n')
    print(wa.api_resp_format,'\n\n\n\n')
